#!/usr/bin/env python
# -*- coding: utf-8 -*-
# $Id$
"""
sicfg.py
Config(s) and command-line processing
"""
from optparse import OptionParser, OptionGroup
from ConfigParser import SafeConfigParser, DuplicateSectionError
from cPickle import dumps, loads, HIGHEST_PROTOCOL
from base64 import encodestring, decodestring
from xml.etree import cElementTree as ET

import sys
import os, os.path
import logging, logging.handlers
import re

from siglobals import *
from siglobals import __default_re

cmdline = OptionParser()
_config_dont_save = [
	'log', 'proxy_user', 'proxy_password', 'dry', 'progress', 'no_check', 'check_inet_only',
	]

_CONFIG_SECTION = 'config'
_XML_ROOT		= 'samizdat' # if ET used istead of config parser




# ===== INIT =====
cmdline.add_option('-?', action='help')

gr = OptionGroup(cmdline, u"Connection", u"Proxy and authorization settings.")
gr.add_option('--proxy','--http-proxy', metavar='PROXY', dest='proxy',
				help=u"Use specified proxy server. Ignore http_proxy environment var if exists.")
gr.add_option('--no-proxy', action='store_true',
				help=u"Don't use any proxies even http_proxy found in env")
gr.add_option('-U', '--proxy-user', metavar='USER',
				help=u"Login for password-protected proxy")
gr.add_option('-P', '--proxy-password', metavar='PASSWORD',
				help=u"Password for password-protected proxy")
gr.add_option('--proxy-auth',
				help=u"Encoded login and password for Basic HTTP authorization")
gr.add_option('--store-auth', action='store_true', dest="store_auth",
				help=u"Allow storing auth information in config file")
gr.add_option('--dont-store-auth', action='store_false', dest="store_auth",
				help=u"Disable storing auth information in config file")

cmdline.add_option_group(gr)


gr = OptionGroup(cmdline, u"Config files")
gr.add_option('--cfg-ignore', choices=('global','local','both'),
				help=u"Eliminate loading default config files. Available choices are: 'global','local','both'.")
gr.add_option('--cfg-file', metavar='FILE', help=u"Load extra configuration from FILE.")
gr.add_option('--cfg-store-local', action="store_const", dest="store_config", const="1",
				help=u"Save current settings as local config file.")
gr.add_option('--cfg-store-global', action="store_const", dest="store_config", const="2",
				help=u"Save current settings as global config file. Note: admin privilegies may be required.")
gr.add_option('--cfg-store', metavar='FILE', help=u"Save current settings into FILE.")
cmdline.add_option_group(gr)


gr = OptionGroup(cmdline, u"Site settings")
gr.add_option('--site', metavar='SITE',
				help=u"Override site name. Default: %s" % (cfg['site'],))
#gr.add_option('--root-page', metavar='PAGE',
#			help=u"Override index page name. Default: %s" % (cfg['root_index'],))
gr.add_option('--charset', metavar='CHARSET',
				help=u"Override site's deafult encoding. Default: %s" % (cfg['default_charset']))
gr.add_option('--allow-if-modified', dest="if_modified", choices=('no','yes'), metavar='yes/no',
				help=u"Enable/disable HTTP If-Modified-Since support")
gr.add_option('--add-stopname', metavar='NAME',
			help=u"Add NAME to stop_names list. The link to NAME will be exclueded from check/download.")
gr.add_option('--remove-stopname', metavar='NAME',
			help=u"Remove NAME to stop_names list. The NAME == '*' clears stop_names list.")

cmdline.add_option_group(gr)

gr = OptionGroup(cmdline, u"Author and url manipulation")
gr.add_option('--add', action="store_true",
				help=u"Add author and/or page from url(s)")
gr.add_option('--add-author', action="store_true",
			help=u"Add only author, not page")
gr.add_option('--remove-url', metavar='URL',
			help=u"Remove book url from database")
gr.add_option('--remove-author', metavar='URL',
			help=u"Remove author and all book urls from database")

gr.add_option('-B', '--base', metavar='FILE_OR_NAME',
			help=u"Database name. Filename for file-based databases")

cmdline.add_option_group(gr)

gr = OptionGroup(cmdline, u"Checking logic")
gr.add_option('--ignore-lastmod', action="store_true",
				help=u"Ignore stored LastModified timestamps")
gr.add_option('--check-csv', action="store_true",
				help=u"Check CSV news only")
gr.add_option('--dry-run', action="store_true",
			help=u"Don't store any data")
gr.add_option('--no-check', action="store_true",
			help=u"Don't check anything. Useful for fast add/remove urls.")
gr.add_option('--check-inet-only', action="store_true",
			help=u"Only check site/internet availability. Exit immedately after test. ")


cmdline.add_option_group(gr)

gr = OptionGroup(cmdline, u"Files and paths")
gr.add_option('--store-enc', metavar='ENCODING',
				help=u"Encoding of locally stored files")
gr.add_option('--store-path', metavar='PATH',
				help=u"Path to store local files")
gr.add_option('--save-local', choices=('no','yes'), metavar='yes/no',
			help=u"Save local copies of modified files")
gr.add_option('--save-bak', choices=('no','yes'), metavar='yes/no',
			help=u"Make backup copies of local files")
gr.add_option('--use-diff', choices=('no','yes'), metavar='yes/no',
			help=u"Use diff command and store differences between ols and new files. (Possibly slow)")
gr.add_option('--save-htm-news', choices=('no','yes'), metavar='yes/no',
			help=u"Generate and save news in HTML format")
gr.add_option('--htm-news-file', metavar='FILE',
			help=u"Filename to store news in HTML format")
#gr.add_option('',
#			help=u"")
cmdline.add_option_group(gr)


gr = OptionGroup(cmdline, u"zhurnal.lib.ru news file")
gr.add_option('--csv-name', metavar='FILENAME',
				help=u"Altername name for .CSV news file on %s. Default: %s" % (cfg['site'], cfg['csv_news']))
gr.add_option('--csv-reload', action="store_true",
				help=u"Ignore timestamp and force download .CSV news")
gr.add_option('--csv-disable', action="store_true",
				help=u"Does not download or use .CSV news")

cmdline.add_option_group(gr)

#gr = OptionGroup(cmdline, u"Database settings")


gr = OptionGroup(cmdline, u"Logging")
gr.add_option('--log', metavar='FILE', dest='log_file',
				help=u"Log to FILE")
gr.add_option('--log-level', metavar='LEVEL', choices=('NOTSET', 'DEBUG', 'INFO', 'WARNING', 'ERROR', 'CRITICAL'),
			help=u"Logging level, default is INFO. Valid choices are: NOTSET, DEBUG, INFO, WARNING, ERROR, CRITICAL" )
gr.add_option('--progress', action="store_true",
			help=u"Print progress information")

#TODO: log-rotate, log-maxsize
#cmdline.set_defaults(log_level='DEBUG')
cmdline.add_option_group(gr)

def load_cfg(filename):
	"""
	Load config from specified path (if exists)
	and apply to global cfg dict
	@type filename: str
	@param filename: name of config file to load
	"""
	global cfg
	try:
		log.debug('Loading config from "%s"' % (filename,))
		cf = ET.ElementTree()
		cf.parse(open(filename, 'rb'))
		p = cf.getroot().find(_CONFIG_SECTION)

		if not p:
			log.debug('\t- No config section.')
			return

		for i in p.getchildren():
			t = i.get('type','str')
			log.debug('\t- get %s(%s)=%s' % (i.tag, t, i.text))
			if t == 'bool':
				if i.text.lower() in ('yes', 'on', 'true','1'): cfg[i.tag] = True
				elif i.text.lower() in ('no', 'off', 'false', '0'): cfg[i.tag] = False
				else:
					log.error(u'Invalid bool value %s="%"' % (i.tag, i.text))
			elif t in ('str','unicode'):
				cfg[i.tag] = i.text
			elif t == 'int':
				try:
					cfg[i.tag] = int(i.text)
				except:
					log.exception(u'Config %s=int(%s) fail.\n' % (i.tag, i.text))
			elif t == 'float':
				try:
					cfg[i.tag] = float(i.text)
				except:
					log.exception(u'Config %s=float(%s) fail.\n' % (i.tag, i.text))
			elif t in ('list','tuple','dict'):
				try:
					_fmt = {'list':u"[%s]", 'tuple':u"(%s)", 'dict':u"{%s}"}
					if i.text: s = i.text.replace('\n','').replace('\r','')
					else: s = ''
					cfg[i.tag] = eval(_fmt[t] % (s,), {}) # disable all locals/globals by '{}'
				except:
					log.exception(u'Config %s=%s(%s) unpack error.\n' % (i.tag, t, i.text))
			elif t == 'pickle':
				s = decodestring(i.text)
				cfg[i.tag] = loads(s)
			else:
				log.error(u'Unknown data type "%s" for %s="%s"' % (t, i.tag, i.text))


	except IOError:
		pass

	except:
		log.exception('Load config from "%s" fail.\n' % (filename,))

def store_cfg(filename):
	"""
	Store config into specified file
	"""
	global cfg
	try:
		log.debug('Storing config into "%s"' % (filename,))

		cf = ET.ElementTree()
		try:
			cf.parse(filename)
			root = cf.getroot()
			p = root.find(_CONFIG_SECTION)
			if p: p.clear()
			else:
				p = ET.Element(_CONFIG_SECTION)
				root.append(p)
		except:
			log.debug('\t- ET parse fail. Ignore')
			root = ET.Element(_XML_ROOT)
			p = ET.Element(_CONFIG_SECTION)
			root.append(p)
			cf._setroot(root)

		p.tail = '\n'
		p.text = '\n'

		for key, val in cfg.iteritems():
			if key.lower() not in _config_dont_save:
				log.debug('\t+ store %s=%s' % (key, val))
				e = ET.Element(key)
				if isinstance(val, (str,unicode,bool,int,float)):
					e.set('type', type(val).__name__)
					e.text = u"%s" % (val,)
				elif isinstance(val, (list,tuple)) and all((isinstance(z,(str,unicode,bool,int,float)) for z in val)):
					e.set('type', type(val).__name__)
					e.text = (u"%s" % (val,))[1:-1] # strip brackets
				elif isinstance(val, dict) and \
						all((isinstance(z,(str,unicode,bool,int,float)) for z in val.keys())) and\
						all((isinstance(z,(str,unicode,bool,int,float)) for z in val.values())):
					e.set('type', type(val).__name__)
					e.text = (u"%s" % (val,))[1:-1] # strip brackets
				elif val == None:
					e.set('type', 'str')
					e.text = ''
				else:
					log.debug('Config: Too complex data. Store "%s" as pickle' % (key,))
					e.set('type','pickle')
					s = encodestring(dumps(val, protocol=HIGHEST_PROTOCOL))
					e.text = s
				e.tail = '\n'
				p.append(e)

		cf.write(open(filename, 'wb'), 'UTF-8')

	except:
		log.exception('Store config into "%s" fail.\n' % (filename,))




def init_log(level, filename=None):
	"""
	initialize logging subsystem.
	"""
	global log
	global cfg
	if not level and not filename: return

	_lvl = {'NOTSET':logging.NOTSET,
			'DEBUG':logging.DEBUG,
			'INFO':logging.INFO,
			'WARNING':logging.WARNING,
			'ERROR':logging.ERROR,
			'CRITICAL':logging.CRITICAL}

	log.setLevel(_lvl.get(level, logging.INFO))
	cfg['log_level'] = logging.getLevelName(log.level)

	#TODO: add rotating handler if needed
	if filename:
		if filename.lower() == 'stderr':
			h = logging.StreamHandler(sys.stderr)
		elif filename.lower() == 'stdout' or filename=='-':
			h = logging.StreamHandler(sys.stdout)
		else:
			h = logging.FileHandler(filename, mode="ab",) # encoding='utf-8')
		f = logging.Formatter("%(asctime)s %(name)-10s %(levelname)-8s: %(message)s", "%d.%m %H:%M:%S")
		h.setFormatter(f)
		log.addHandler(h)

	cfg['log'] = log



def encode_auth(username, password):
	"""
	Encode plain username and password as Basic auth argument.

	>>> encode_auth('vasya_pupkin','FoReVeR')
	'dmFzeWFfcHVwa2luOkZvUmVWZVI='
	"""
	return encodestring('%s:%s' % (username, password)).replace('\n','')

def process_cmdline(opt, arg):
	"""
	decode command-line argumens and store into cfg dict
	"""
	# proxy group
	if opt.proxy: cfg['proxy'] = opt.proxy
	if opt.no_proxy: cfg['no_proxy'] = True
	if opt.proxy_auth: cfg['proxy_auth'] = opt.proxy_auth
	if opt.proxy_user:
		if opt.proxy_password: cfg['proxy_auth'] = encode_auth(opt.proxy_user, opt.proxy_password)
		else:
			log.warning('Proxy password not found.')
			cfg['proxy_auth'] = encode_auth(opt.proxy_user, '')
	if opt.store_auth != None :cfg['store_auth'] = bool(opt.store_auth)
	if not cfg['store_auth']: _config_dont_save.append('proxy_auth')


	# csv group
	if opt.csv_name: cfg['csv_news'] = opt.csv_name
	if opt.csv_reload: cfg['csv_reload'] = True
	if opt.csv_disable: cfg['csv_disable'] = True

	# site group
	if opt.site: cfg['site'] = opt.site
	if opt.charset: cfg['default_charset'] = opt.charset
	if opt.if_modified: cfg['if_modified_supported'] = opt.if_modified == 'yes'
	if opt.remove_stopname:
		try:
			if opt.remove_stopname in cfg['stop_names']:
				cfg['stop_names'] = list(cfg['stop_names']) # tuple/paranoic protection
				cfg['stop_names'].pop(cfg['stop_names'].index(opt.remove_stopname))
			elif opt.remove_stopname == '*':
				log.debug(u'--remove-stopname=*. Clear stop_names!')
				cfg['stop_names'] = []
		except ValueError:
			log.debug(u'--remove-stopname=%s. Already removed.' % (opt.remove_stopname,))
		except:
			log.exception(u'--remove-stopname=%s. Error:\n' % (opt.remove_stopname,))
	if opt.add_stopname:
		try:
			if opt.add_stopname not in cfg['stop_names']:
				cfg['stop_names'] = list(cfg['stop_names'])
				cfg['stop_names'].append(opt.add_stopname)
		except:
			log.exception(u'--add-stopname=%s. Error:\n' % (opt.add_stopname,))

	# files group
	if opt.store_path: cfg['store_path'] = opt.store_path
	if opt.save_local: cfg['store_local'] = opt.save_local=='yes'
	if opt.save_bak: cfg['store_bak'] = opt.save_bak=='yes'
	if opt.use_diff:cfg['use_diff'] = opt.use_diff=='yes'
	if opt.store_enc: cfg['store_enc'] = opt.store_enc
	if opt.save_htm_news: cfg['store_htmnews'] = opt.save_htm_news=='yes'
	if opt.htm_news_file: cfg['news_htm'] = opt.htm_news_file

	# author and url group
	if opt.add:
		cfg['add'] = True
		_config_dont_save.append('add')
	if opt.add_author:
		cfg['add_author'] = True
		_config_dont_save.append('add_author')

	if opt.remove_url:
		cfg['remove_url'] = opt.remove_url
		_config_dont_save.append('remove_url')
	if opt.remove_author:
		cfg['remove_author'] = opt.remove_author
		_config_dont_save.append('remove_author')

	if opt.base: cfg['base'] = opt.base

	# check group
	if opt.ignore_lastmod: cfg['ignore_lastmod'] = True
	if opt.check_csv: cfg['csv_only'] = True
	if opt.dry_run: cfg['dry'] = True
	if opt.no_check: cfg['no_check'] = True
	if opt.check_inet_only: cfg['check_inet_only'] = True

	# logging group
	if opt.progress == True: cfg['progress'] = True


def init_re():
	"""
	precompile regexps and add into cfg[] on success
	"""

	for key,val in cfg.iteritems():
		if not key.startswith('re_'): continue
		try:
			if isinstance(val, (tuple, list)):
				r = re.compile(val[0], val[1])
			else:
				r = re.compile(val, re.I | re.U)
			regexp[key] = r
		except:
			log.warning(u'Unable to compile regexp %s="%s". Default will be used.' % (key, val))
			if key in __default_re:
				try:
					if isinstance(val, (tuple, list)):
						r = re.compile(__default_re[key][0], __default_re[key][1])
					else:
						r = re.compile(__default_re[key], re.I | re.U)
					regexp[key] = r
				except:
					log.exception('init_re fail.')


def init(sys_argv=None):
	"""
	Process command-line arguments and load config(s)

	"""
	global_config_name = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'default.conf')
	local_config_name = os.path.join(os.getenv('HOME',
						os.getenv('APPDATA',
							os.getenv('USERPROFILE',
								os.path.dirname(os.path.abspath(__file__))))),
						('.' if os.name not in ('nt', 'ce') else '') + 'pysamizdat', 'local.conf')

	opt, arg = cmdline.parse_args(sys_argv or sys.argv[1:])

	## 1. Set logging level if passed
	#
	if opt.log_file or opt.log_level or cfg['log_file']:
		init_log(opt.log_level or cfg['log_level'], opt.log_file or cfg['log_file'])

	## 2. Load config files if allowed
	#
	if opt.cfg_ignore not in ('global', 'both'):
		load_cfg(global_config_name)
	if opt.cfg_ignore not in ('local', 'both'):
		load_cfg(local_config_name)
	if opt.cfg_file:
		load_cfg(opt.cfg_file)

	# restore log level
	cfg['log_level'] = logging.getLevelName(cfg['log'].level) if cfg['log'] else 'DEBUG'

	## 3. Apply rest of cmdline
	#
	process_cmdline(opt, arg)
	init_re()	# check/generate regexp values

	## 4. Try store config if passed by cmdline
	#

	###TEST###
	if 'save_count' in cfg:
		cfg['save_count'] += 1
	###    ###

	if opt.store_config == "2":
		store_cfg(global_config_name)
	elif opt.store_config == "1":
		store_cfg(local_config_name)
	if opt.cfg_store:
		store_cfg(opt.cfg_store)


	return arg



if __name__ == "__main__":
	#import logging
	log.addHandler(logging.StreamHandler())
	log.setLevel(logging.DEBUG)
	log.info("%s Unittests." % (__file__,))

	log.info("=====DOCTESTS=====")
	import doctest
	doctest.testmod()
	log.info("===END DOCTESTS===")

	'''
	cfg['unicode_test'] = u"Проверка UNICODE текста"
	cfg['int_test'] = 12345
	cfg['float_test'] = 12.345
	cfg['save_count'] = 0
	cfg['tuple_test'] = (1,'aaa', 'tuple_test')
	cfg['dict_test'] = {'dict':'value',11:'numeric_dict'}
	'''
	init()
